/*
 * @Author: zi.yang
 * @Date: 2023-11-24 13:02:39
 * @LastEditors: zi.yang
 * @LastEditTime: 2023-11-26 22:58:40
 * @Description: 爬取豆瓣电影 top250
 * @FilePath: /node-classical/index.js
 */
import fs from 'node:fs';
import { promisify } from 'node:util';

import got, { Options } from 'got';
import jsdom from 'jsdom';

const userAgents = [
  "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
  "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
  "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
  "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0) ,Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9",
  "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
  "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
  "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
  "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
  "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12",
  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
  "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
  "Opera/9.25 (Windows NT 5.1; U; en), Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9",
  "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'
];

let proxyIpList = [];

// 获取 dom
const getDocument = (html) => new jsdom.JSDOM(html).window.document;
// 随机数函数
const randomInt = (min, max) => Math.floor(Math.random() * max + min)

const getProxyIp = async () => {
  const page = await got.get('http://api.89ip.cn/tqdl.html?api=1&num=60&port=&address=&isp=');
  const dom = getDocument(page.body)
  const ipList = dom.body.innerHTML.match(/(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}:\d+)/g)
  return Array.from(ipList);
}

// 构造请求方法
const fetchPage = (start) => {
  const options = new Options({
    prefixUrl: 'https://movie.douban.com/',
    searchParams: { start },
    headers: {
      'User-Agent': userAgents[randomInt(0, userAgents.length - 1)],
      'x-forwarded-for': proxyIpList[randomInt(0, proxyIpList.length - 1)],
      // 'Cookie': ``
    },
  })
  return got.get(`top250`, options)
}
const writeFile = promisify(fs.writeFile);

async function getPageContent(page) {
  try {
    const result = await fetchPage(page)
    const dom = getDocument(result.body);
    const movieList = dom.querySelectorAll('#content .grid_view li')
    const content = [];
    movieList.forEach((element) => {
      const title = element.querySelector('.info .title')?.textContent || '';
      const ratingNum = element.querySelector('.bd .rating_num')?.textContent || '';
      const quote = element.querySelector('.quote > .inq')?.textContent || '';
      const number = element.querySelector('.bd span:last-child')?.textContent || ''
      const img = element.querySelector('.pic img').getAttribute('src')
      content.push({ title, ratingNum, quote, number, img })
    })
    return content;
  } catch (error) {
    console.error(error)
    return []
  }
}

const sleep = (time) => new Promise(resolve => setTimeout(resolve, time))

async function main() {
  const content = [];
  for (let page = 0; page < 10; page++) {
    if (page > 0) await sleep(randomInt(1000, 3000))
    const result = await getPageContent(page * 25);
    if (!result.length) page--;
    content.push(...result);
  }
  if (content.length !== 250) {
    console.log(`内容未全部爬取，当前爬取总数：${content.length}`)
  }
  await writeFile('./content.json', JSON.stringify(content), { encoding: 'utf-8' })
}

try {
  proxyIpList = await getProxyIp();
  console.log('爬虫程序开始执行，请稍后...')
  const logId = '爬虫运行总计时：'
  console.time(logId);
  await main();
  console.timeEnd(logId);
  console.log('爬虫程序执行完成！')
} catch (error) {
  throw new Error('[ NODE-CLASSICAL ]: Main 函数运行异常！\n' + error)
}
